# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.pipelines.images import ImagesPipeline
import codecs
import json
from scrapy.exporters import JsonItemExporter
import  MySQLdb
from robot.utils.common import DateEncoder
# 2. 主要用于数据存储,在接收到item之后可以进行相应的操作,先执行哪一个item,由settings.py进行设置
class RobotPipeline(object):
    def process_item(self, item, spider):
        # 主要用于存储数据库操作
        return item

class JsonWithEncodingPipeline(object):
    # 自定义json文件的导出
    def __init__(self):
        self.file = codecs.open('article.json','w',encoding="utf-8")

    def process_item(self,item,spider):
        # item转换为dict，字典
        lines = json.dumps(dict(item),ensure_ascii=False)+"\n"
        self.file.write(lines)
        return item

    def spider_closed(self,spider):
        self.file.close()

class MySQLPipeline(object):
    def __init__(self):
        self.conn = MySQLdb.connect(host="127.0.0.1",user="root",password="123",db="scrapy",port=3306,charset="utf8",use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        print("hello")
        # 执行sql语句
        insert_sql = """
        insert into jobbole_article(title,url,create_date,url_object_id,front_image_url,front_image_path,fav_nums,praise_nums,comment_nums,tags,content) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        self.cursor.execute(insert_sql,(item["title"],item["url"],item["create_date"],item["url_object_id"],item["front_image_url"],item["front_image_path"],item["fav_nums"],item["praise_nums"],item["comment_nums"],item["tags"],item["content"]))

class JsonExportPipeline(object):
    # 调用scrapy提供的json export到处json文件
    def __init__(self):
        self.file = codecs.open('articleexport.json','wb')
        self.exporter = JsonItemExporter(self.file,encoding="utf-8",ensure_ascii=False)
        self.exporter.start_exporting()

    def close_spider(self,spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self,item,spider):
        # item转换为dict，字典
        self.exporter.export_item(item)
        return item

# 1. 自定义pipeline将本地保存图片的路径进行获取
class ArticleImagePipleline(ImagesPipeline):
    def item_completed(self, results, item, info):
        for ok,value  in results:
            item["front_image_path"] = value["path"]
        # 此处需要将item返回，交给其他的pipline进行处理
        return item